package cn.keyvalue.ccf.common.util;

import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.UrlValidator;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.visitors.HtmlPage;

import cn.keyvalue.ccf.common.http.NewRequestUtils;
import cn.keyvalue.ccf.common.object.Hyperlink;
import cn.keyvalue.ccf.common.object.Page;

/**
 * HTML文件解析工具集
 * 
 * @author wangxinning
 * 
 */
public class HtmlUtils {

	/**
	 * 解析html页面中的html
	 * 
	 * pageString必须是标准html格式或者一段完整的标签字符串
	 * urlStr是该字符串的来源url，可以解决如果是相对地址的链接，自动补齐域名信息
	 * 
	 * @param urlStr
	 * @param pageString
	 * @return
	 * @throws ParserException
	 * @throws MalformedURLException
	 * @throws URISyntaxException
	 */
	public static List<Hyperlink> parseAllLinkAndLinkName(String sourceUrl,
			String pageString) throws ParserException, MalformedURLException,
			URISyntaxException {
		List<Hyperlink> retList = new ArrayList<Hyperlink>();

		URL u = new URL(sourceUrl);

		String host = "http://" + u.getHost();
		String path = StringUtils.trimToEmpty(u.getPath());

		if (StringUtils.contains(path, "/")) {
			path = StringUtils.substringBeforeLast(path, "/");
		}
		if (StringUtils.isEmpty(path)) {
			path = "/";
		}

		// 规范html格式
		pageString = formatHtml( pageString );

		Parser parser = new Parser(pageString);
		HtmlPage page = new HtmlPage(parser);
		parser.visitAllNodesWith(page);

		NodeList nodelist = page.getBody();
		NodeFilter filter = new TagNameFilter("A");
		nodelist = nodelist.extractAllNodesThatMatch(filter, true);

		UrlValidator urlValidator = new UrlValidator();
		for (int i = 0; i < nodelist.size(); i++) {
			LinkTag link = (LinkTag) nodelist.elementAt(i);
			
			//href属性必须存在
			String url = link.getAttribute("href");
			if( url == null )
			{
				continue;
			}
			
			
			String title = StringUtils.trimToEmpty(link.getAttribute("title"));
			url = StringUtils.replace(StringUtils.trimToEmpty(url),"&amp;","&");
			String value = StringUtils.trimToEmpty(link.getLinkText());
						
			if (StringUtils.startsWith(url, "/")) {
				url = host + url;
			} else if (!StringUtils.startsWith(url, "http://")) {
				url = host + path + url;
			}

			if (urlValidator.isValid(url) && StringUtils.isNotEmpty(value)
					&& !StringUtils.contains(url, "(")
					&& !StringUtils.contains(url, "javascript")
					&& !StringUtils.contains(url, "'")
					&& !StringUtils.contains(url, "\"")
					&& !StringUtils.startsWith(url, "https")) {
				if (!retList.contains(url)) {

					Hyperlink hyperlink = new Hyperlink();
					hyperlink.setUrl(url);
					hyperlink.setName(value);
					hyperlink.setTitle(title);
					retList.add(hyperlink);
				}
			}
		}

		return retList;
	}

	public static String parseHtmlString(String htmlString, Class class1,
			String attributeName, String attributeValue) {
		Parser parser;
		try {
			parser = new Parser(htmlString);

			NodeFilter filter = new NodeClassFilter(class1);
			HasAttributeFilter hasAttributeFilter = new HasAttributeFilter();
			hasAttributeFilter.setAttributeName(attributeName);
			hasAttributeFilter.setAttributeValue(attributeValue);

			NodeList list = parser.extractAllNodesThatMatch(new AndFilter(
					filter, hasAttributeFilter));

			if (list != null && list.size() > 0) {

				return list.elementAt(0).toHtml();

			}
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}

	public static String parseHtmlString(String htmlString, String tag,
			String attributeName, String attributeValue) {
		Parser parser;
		try {
			parser = new Parser(htmlString);
			NodeFilter filter = new TagNameFilter(tag);
			// NodeFilter filter = new NodeClassFilter(class1);
			HasAttributeFilter hasAttributeFilter = new HasAttributeFilter();
			hasAttributeFilter.setAttributeName(attributeName);
			hasAttributeFilter.setAttributeValue(attributeValue);

			NodeList list = parser.extractAllNodesThatMatch(new AndFilter(
					filter, hasAttributeFilter));

			if (list != null && list.size() > 0) {

				return list.elementAt(0).toHtml();

			}
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}
	
	
	public static String parseHtmlString(String htmlString, String tag) {
		Parser parser;
		try {
			parser = new Parser(formatHtml(htmlString));
			NodeFilter filter = new TagNameFilter(tag);

			NodeList list = parser.extractAllNodesThatMatch(filter);

			if (list != null && list.size() > 0) {

				return list.elementAt(0).toHtml();

			}
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}
	
	public static String[] parseHtmlArray(String htmlString, String tag,String includekeyword) {
		Parser parser;
		try {
			parser = new Parser(formatHtml(htmlString));

			NodeFilter filter = new TagNameFilter(tag);

			NodeList list = parser.extractAllNodesThatMatch(filter);

			String[] retArray = null;
			for (int i = 0; i < list.size(); i++) {
				if (StringUtils.contains(list.elementAt(i).toHtml(),
						includekeyword) || StringUtils.isEmpty(includekeyword)) {
					retArray = (String[]) ArrayUtils.add(retArray, list
							.elementAt(i).toHtml());
				}
			}

			return retArray;
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}
	
	public static String[] parseHtmlArray(String htmlString, Class class1,
			String attributeName, String attributeValue, String includekeyword) {
		Parser parser;
		try {
			parser = new Parser(htmlString);

			NodeFilter filter = new NodeClassFilter(class1);
			HasAttributeFilter hasAttributeFilter = new HasAttributeFilter();
			hasAttributeFilter.setAttributeName(attributeName);
			hasAttributeFilter.setAttributeValue(attributeValue);

			NodeList list = null;
			if (StringUtils.isEmpty(attributeName)) {
				list = parser.extractAllNodesThatMatch(filter);
			} else {
				list = parser.extractAllNodesThatMatch(new AndFilter(filter,
						hasAttributeFilter));
			}

			String[] retArray = null;
			for (int i = 0; i < list.size(); i++) {
				if (StringUtils.contains(list.elementAt(i).toHtml(),
						includekeyword) || StringUtils.isEmpty(includekeyword)) {
					retArray = (String[]) ArrayUtils.add(retArray, list
							.elementAt(i).toHtml());
				}
			}

			return retArray;
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}

	public static String[] parseHtmlArray(String htmlString, String tag,
			String attributeName, String attributeValue, String includekeyword) {
		Parser parser;
		try {
			parser = new Parser(htmlString);

			NodeFilter filter = new TagNameFilter(tag);
			// NodeFilter filter = new NodeClassFilter(class1);
			HasAttributeFilter hasAttributeFilter = new HasAttributeFilter();
			hasAttributeFilter.setAttributeName(attributeName);
			hasAttributeFilter.setAttributeValue(attributeValue);

			NodeList list = null;
			if (StringUtils.isEmpty(attributeName)) {
				list = parser.extractAllNodesThatMatch(filter);
			} else {
				list = parser.extractAllNodesThatMatch(new AndFilter(filter,
						hasAttributeFilter));
			}

			String[] retArray = null;
			for (int i = 0; i < list.size(); i++) {
				if (StringUtils.contains(list.elementAt(i).toHtml(),
						includekeyword) || StringUtils.isEmpty(includekeyword)) {
					retArray = (String[]) ArrayUtils.add(retArray, list
							.elementAt(i).toHtml());
				}
			}

			return retArray;
		} catch (Exception e) {
			e.printStackTrace();
		}

		return null;
	}
	
	
	
	public static Page parsePage( String url ) throws Exception
	{
		String ua = NewRequestUtils.getRandomUserAgent();
		HttpClient client = NewRequestUtils.getHttpClient(ua, 5000, 5000);
		String content = NewRequestUtils.urlToStr(client, url,
				NewRequestUtils.METHOD_GET);

		NewRequestUtils.closeHttpClient(client);
		
		Page page = parsePageByString( content );
		page.setUrl(url);
		return page;
	}
	
	
	public static String formatPageInfo( String str ) {
		
		return StringUtils.replace(StringUtils.trimToEmpty(str),"\n","");
		
	}
	
	
	public static Page parsePageByString( String htmlString )
	{
		Page page = new Page();
		String title = StringUtils.substringBetween(htmlString, "<title>", "</title>");
		
		//<meta  name="description" content="易迅网在线销售笔记本电脑，并为你购买笔记本电脑提供最新报价、用户评价、图片、促销等选购信息" />

		String tempString = parseHtmlString(htmlString,"meta", "name", "description");
		if( StringUtils.isBlank(tempString) )
		{
			tempString = parseHtmlString(htmlString,"meta", "name", "Description");
		}
		
		//System.out.println( tempString );
				
		//String tempdescription = StringUtils.deleteWhitespace(StringUtils.substringBetween(htmlString, "description", ">"));
		String description = StringUtils.substringBetween(tempString, "content=\"", "\"");		
		
		if( StringUtils.isBlank(description) )
		{
			description = StringUtils.substringBetween(tempString, "content='", "'");
		}
		
		page.setTitle(formatPageInfo(title));
		page.setDescription(formatPageInfo(description));
		
		return page;
	}
	
	
	/**
	 * 从URL或者URL的一部分中获取参数值
	 * @param urlorquary
	 * @param name
	 * @return
	 */
	public static String getParameterValue( String urlorquary, String name )
	{
		if( StringUtils.containsIgnoreCase(urlorquary, name+"=") )
		{
			//System.out.println( urlorquary );
			String str = StringUtils.replace(urlorquary, name+"=", StringUtils.lowerCase(name)+"=");
			//System.out.println( name+"=" );
			//System.out.println( StringUtils.lowerCase(name)+"=" );
			String retStr = StringUtils.substringBetween(str, StringUtils.lowerCase(name)+"=", "&");
			
			if( StringUtils.isEmpty(retStr) )
			{
				retStr = StringUtils.substringAfter(str, StringUtils.lowerCase(name)+"=");
			}
			
			return retStr;			
		}
		
		return null;
	}
	
	
	
	public static String removeParameter( String urlorquary, String name )
	{
		if( StringUtils.contains(urlorquary, name + "=") )
		{
			String value = StringUtils.substringBetween(urlorquary, name + "=", "&");
			if( StringUtils.isEmpty(value) )
			{
				value = StringUtils.substringAfter(urlorquary, name + "=");
			}
			
			String retStr = urlorquary;
			retStr = StringUtils.remove(retStr, name + "=" + value + "&");
			retStr = StringUtils.remove(retStr, "&" + name + "=" + value);
			retStr = StringUtils.remove(retStr, "?" + name + "=" + value);
			
			return retStr;
			
		}
		
		return urlorquary;
	}
	
	
	
	public static String addParameter( String urlorquary, String name, String value )
	{
		String retStr = urlorquary;
		if( StringUtils.contains(retStr, name + "=") )
		{
			retStr = removeParameter(retStr, name);
		}
		
		if( StringUtils.contains(retStr, "?") )
		{
			retStr = retStr + "&" + name + "=" + value;
		}
		else
		{
			retStr = retStr + "?" + name + "=" + value;
		}
		
		return retStr;
	}
	
	
	public static String formatHtml( String pageString )
	{
		if (!StringUtils.contains(pageString, "<html")
				&& !StringUtils.contains(pageString, "<body")) {
			String aaString = "<html><head></head><body>" + pageString
					+ "</body></html>";
			
			return aaString;			
		}
		else
		{
			return pageString;
		}
	}
	
	
	public static void main( String[] args ) throws Exception
	{
		String url = "http://list.51buy.com/496--------.html";
		
		//System.out.println(StringUtils.remove(l, "?"));
		System.out.println(HtmlUtils.parsePage(url));
	}
}